import bsddb
import struct
import json
import sys
import random
import time

class MikeWebgraph:
    def __init__(self,dataset):
        start_t = time.time()
        self.dataset = dataset
        self.urlid = bsddb.btopen(dataset+'/urlid.bdb','r')
        self.idurl = bsddb.btopen(dataset+'/idurl.bdb','r')
        self.webgraph = open(dataset+'/webgraph.bin','rb').read()
        self.transposegraph = open(dataset+'/transposegraph.bin','rb').read()
        self.hostgraph = open(dataset+'/hostgraph.bin','rb').read()
        self.statdbpath = dataset+'/stat.json'
        self.webgraphindex = {}
        self.transposegraphindex = {}
        self.hostgraphindex = {}
        #read webgraph
        pos = 0
        while 1:
            try:
                src = int(struct.unpack_from('i',self.webgraph,pos)[0])
                self.webgraphindex[src] = int(pos)
                pos += 4
                outdegree = struct.unpack_from('i',self.webgraph,pos)[0]
                pos += 4*(outdegree+1)
            except:
                break
        #read transposegraph 
        pos = 0
        while 1:
            try:
                src = struct.unpack_from('i',self.transposegraph,pos)[0]
                self.transposegraphindex[src] = pos
                pos += 4
                indegree = struct.unpack_from('i',self.transposegraph,pos)[0]
                pos += 4*(indegree+1)
            except:
                break
        #read hostgraph
        pos = 0 
        while 1:
            try:
                src = int(struct.unpack_from('i',self.hostgraph,pos)[0])
                self.hostgraphindex[src] = pos
                pos += 4
                total = struct.unpack_from('i',self.hostgraph,pos)[0]
                pos += 4*(total+1)
            except:
                break
        # create danglingnode list
        self.src_list = self.list('source','all')
        self.danglingnode = list(set([i for i in range(int(self.stat()['total_unique_url']))])-set(self.src_list))
        start_t = time.time()-start_t
        print 'Service Available in',start_t

    def forward(self,srcid):
        ans = {}
        if self.webgraphindex.has_key(int(srcid)) :
            startpos = self.webgraphindex[srcid]
            pos = startpos + 4
            outdegree = struct.unpack_from('i',self.webgraph,pos)[0]
            urllist = []
            for i in range(outdegree):
                destid = struct.unpack_from('i',self.webgraph,pos)[0]
                urllist.append(int(destid))
                pos += 4
            return urllist
        else:
            return None

    def backward(self,urlrequest):
        ans = {}
        if self.urlid.has_key(urlrequest):
            destid = int(self.urlid[urlrequest])
            startpos = self.transposegraphindex[destid]
            indegree = int(struct.unpack_from('i',self.transposegraph,startpos+4)[0])
            urllist = []
            pos = startpos
            for i in range(int(indegree)):
                srcid = struct.unpack_from('i',self.transposegraph,pos)[0]
                urllist.append(int(srcid))
                pos += 4
            return urllist
        else:
            return None

    def site(self,urlrequest):
        ans = {}
        if self.urlid.has_key(urlrequest):
            urllist = []
            hostid = int(self.urlid[urlrequest]) 
            if self.hostgraphindex.has_key(hostid):
                startpos = self.hostgraphindex[hostid]
                total = int(struct.unpack_from('i',self.hostgraph,startpos+4)[0])
                pos = startpos
                if total != 0:
                    for i in range(total):
                        srcid = struct.unpack_from('i',self.hostgraph,pos)[0]
                        urllist.append(self.idurl[str(srcid)])
                        pos += 4
                ans['urlid'] = hostid
                ans['count'] = total
                ans['result'] = urllist
                ans['urlrequest'] = urlrequest
            else:
                ans['error'] = True
                ans['message'] = 'Either URL not found or not match'
        else:
            ans['error'] = True
            ans['message'] = 'Either URL not found or not match'

        return json.dumps(ans)

    def list(self,value,maximum):
        ans = {}
        if maximum == 'all' :
            popularity = []
            if value == 'source' :
                popularity = self.webgraphindex.keys()
            elif value == 'destination' :
                popularity = self.transposegraphindex.keys()
            return popularity
        else:
            maximum = int(maximum)
            if value == 'source' :
                if maximum < len(popularity):
                    sample = random.sample(popularity,maximum)
                    return sample
                else:
                    return None
            elif value == 'destination' :
                popularity = self.transposegraphindex.keys()
                if maximum < len(popularity):
                    sample = random.sample(popularity,maximum)
                    return sample
                else:
                    return None
    
    def urltoid(self,url):
        ans = {}
        if self.urlid.has_key(url) :
            return self.urlid[url]
        else:
            return None

    def idtourl(self,id):
        ans = {}
        if self.idurl.has_key(str(id)) :
            return self.idurl[str(id)]
        else:
            return None

    def stat(self):
        statdb = json.loads(open(self.dataset+'/stat.json','r').read())
        return statdb

    def close(self):
        self.idurl.close()
        self.urlid.close()

if __name__ == '__main__' :
    #wg = MikeWebgraph('th-test')
    #a = len(wg.src_list)
    #b = len(wg.danglingnode)
    #print a+b
    #print wg.stat()['total_unique_url']
    pass
